#coding:utf-8

import scrapy
from ..items import TencentItem, PositionItem, AllItem


class TencentSpider(scrapy.Spider):
    #scrapy genspider tencent hr.tencent.com

    #Sipder类中__init__()方法 初始化了爬虫名字和start_url列表,若在继承Spider类时重写__init__()方法,则需要继承父类的该方法
    # super(TencentSpider, self).__init__(name, **kwargs)

    name = "tencent2"

    allowed_domains = ["hr.tencent.com"]

    # python3可能会出现无法取到类属性，所以通过 global
    #global base_url, page
    #base_url = "https://hr.tencent.com/position.php?&start="
    #start_urls = [base_url + str(page)]

    start_urls = ["https://hr.tencent.com/position.php?&start=" + str(page) for page in range(0, 2851, 10)]


    def parse(self, response):
        """
            解析列表页的响应数据
        """
        node_list = response.xpath("//tr[@class='even'] | //tr[@class='odd']")

        for node in node_list:
            item = AllItem()
            # 职位名
            item['position_name'] = node.xpath("./td[1]/a/text()").extract_first()
            # 详情链接
            item['position_link'] = "https://hr.tencent.com/" + node.xpath("./td[1]/a/@href").extract_first()
            # 职位类型
            item['position_type'] = node.xpath("./td[2]/text()").extract_first()
            # 招聘人数
            item['people_number'] = node.xpath("./td[3]/text()").extract_first()
            # 工作地点
            item['work_location'] = node.xpath("./td[4]/text()").extract_first()
            # 发布时间
            item['publish_times'] = node.xpath("./td[5]/text()").extract_first()

            # 这里需注意: 既然是放在一个item里,那么只需在处理完item后yield就行了,如果这里yield item 那么就多存了列表页数据

            # 每次迭代提取一组数据item，通过yield 交给引擎-管道
            #yield item

            # 职位详情页的请求
            #meta属性用来传递当前方法里的数据,最后会作为响应的meta参数传递到callback的回调函数里
            yield scrapy.Request(url=item['position_link'], callback=self.parse_detail, meta={"item": item})

            # link_extractor = LinkExtractor(allow=r'position_detail\.php\?id=\d+')
            # links = link_extractor.extract_links(response)
            #
            # for link in links:
            #     yield scrapy.Request(link.url, callback=self.parse_detail)


    def parse_detail(self, response):
        """
            详情页的响应提取
        """
        item = response.meta["item"]

        item['position_zhize'] = response.xpath("//ul[@class='squareli']")[0].xpath("./li/text()").extract()

        item['position_yaoqiu'] = response.xpath("//ul[@class='squareli']")[1].xpath("./li/text()").extract()

        yield item
        #return [item]  # 这里当只有一个yield返回迭代数据时,可以采用return返回列表,但若是多个就不行

        """
        而如果还有第三级页面,那么还构造request
        yield scrapy.Request(next_url, callback=self.parse_next, meta=response.meta)

        然后再定义parse_next()方法

        """


"""
scrapy.Request构造了请求对象并发起请求返回响应后执行回调函数,并携带参数meta

这里大致上描述下request和response的构造

class Request(object):
    def __init__(self, url, callback, meta):
        self.url = url
        self.callback = callback, meta
        self.meta = meta

class Response(object):
    def __init__(self, url, status, headers, meta):
        self.meta = request.meta

"""